#ifndef CTOKENIZER_H_
#define CTOKENIZER_H_

#include <fstream>
#include <map>
#include <vector>
#include "ctoken.h"
#include "../../exception/cfilenotexistexception.h"

/**
 * CTokenizer - class with methods resolving file into tokens
 * @author Michal Rost
 * @date 26.04.2008
 */
class CTokenizer : public CObject {
public:
  static vector<CToken> tokenize(string fileName,
      const map<string, int> &keywords, bool skipComments = true,
      bool skipWhiteSpaces = true, bool skipEndOfLines = true,
      bool skipTabulators = true);
  static string getClassName() {
    return "CTokenizer";
  }
protected:
  static CToken getToken(ifstream &fh, int line,
      const map<string, int> &keywords);
  static CToken getStringExpression(ifstream &fh, string q);
  static CToken getCommentExpression(ifstream &fh);
  static map<string, int>::const_iterator matchKeyword(string str,
      TIndex &pos, const map<string, int> &keywords);
};

#endif
